#ifndef __PAGELIB_H__
#define __PAGELIB_H__

#include "DirScanner.h"
#include "WebPage.h"
#include "PageLibPreprocessor.h"
#include "InvertIndexMaker.h"
#include "WordSegmentation.h"
#include <vector>
#include <map>
#include <string>
#include <unordered_set>
#include <unordered_map>

using std::vector;
using std::string;
using std::map;
using std::unordered_set;
using std::unordered_map;

struct RssItem
{
    string title;
    string url;
    string content;
};

struct Equal_to {
    bool operator()(const uint64_t& lhs, const uint64_t& rhs) const {
        return PageLibPreprocessor::getInstance()->isSame(lhs, rhs, 3);
    }
};

class PageLib
{
public:

    PageLib(const string &dirPath);
    //创建去重前网页库和网页偏移库
    void createOldPageLib();
    //存储去重后网页库和偏移库
    void storeNewPageLib();
    //存储倒排索引库
    void storeInvertIndexTable();
    //去除html标签
    string mv_tag(const string &s);
    //网页去重
    void cutRedundantPages(); 
    //加载停用词
    void loadStopWords();
    //将网页分词并统计各网页的词频
    void makeWebWordsFreq();
    
    
private:
    DirScanner _dirScanner;  //扫描目录获得语料文件路径
    vector<WebPage> _webPages; //去重前网页库对象
    vector<WebPage> _webPagesNew; //去重后网页库对象
    //map<size_t, std::pair<size_t, size_t>> _offsetLib;  //去重前偏移库对象
    unordered_set<uint64_t, std::hash<uint64_t>, Equal_to> _simHashs;
    unordered_map<string, unordered_map<int, double>> _invertIndexTable;
    InvertIndexMaker _invertIndexMaker;   //倒排索引生成类

    unordered_set<string> _stopWords; //存储停用词
    WordSegmentation _wordSeg; //分词器
};

#endif